<?php

#替换抓来的文字

function spiderReplaceContent($content) {
    if (empty($GLOBALS['varStore']['spiderReplaceContent'])) {
        if (!file_exists(SPIDER_REPLACE_LIST_TXT_FILE)) {
            log::error('SPIDER_REPLACE_LIST_TXT_FILE No Such File!');
            die();
        }
        $GLOBALS['varStore']['spiderReplaceContent'] = file(SPIDER_REPLACE_LIST_TXT_FILE);
        foreach ($GLOBALS['varStore']['spiderReplaceContent'] as &$v) {
            $v = trim($v);
        }unset($v);
    }
    return str_replace($GLOBALS['varStore']['spiderReplaceContent'], '', $content);
}

#提取标题和正文

function spiderFetchHtml($url) {
    global $dhttp_response_header;
    $http_response_header = $dhttp_response_header;
    global $dhttp_lastredirectaddr;

    $content = snoopy_fetch($url);
    $trueUrl = $url;
    $uriinfo = parse_url($trueUrl);
    if ($trueUrl == 'http://www.' . $uriinfo['host'] . '/' || $trueUrl == 'http://' . $uriinfo['host'] . '/' ||
            $trueUrl == 'http://www.' . $uriinfo['host'] || $trueUrl == 'http://' . $uriinfo['host']) {
        log::dw('Domain == Event,May be Index Default:' . $trueUrl);
        return false;
    }

    $uriinfo = parse_url($dhttp_lastredirectaddr);
    if ($trueUrl == 'http://www.' . $uriinfo['host'] . '/' || $trueUrl == 'http://' . $uriinfo['host'] . '/' ||
            $trueUrl == 'http://www.' . $uriinfo['host'] || $trueUrl == 'http://' . $uriinfo['host'] ||
            $trueUrl == 'http://www.' . $uriinfo['host'] . '/index.html' || $trueUrl == 'http://' . $uriinfo['host'] . '/index.html') {
        log::dw('Content-Location Event,May be Index Default:' . $trueUrl);
        return false;
    }

    $host = $uriinfo['host'];
    foreach ($GLOBALS['varStore']['pingBiDomainList'] as $tHost) {
        if (strpos($host, $tHost)) {
            log::dw('The Domain Host is Forbidden in Config File! Hostname:' . $host);
            return false;
        }
    }

    $text = HTMLExtractor::getUrlMainContent($url, 200, 1, $content);
    $text = HTMLExtractor::convertToUTF8($text);
    $text = str_replace('&nbsp;', '', $text);
    $text = str_replace("\n", '<br />', $text);
    $text = preg_replace("#\s+#", '<br />', $text);
    $text = str_replace("<br /><br />", '<br />', $text);
    $text = str_replace("<br /><br />", '<br />', $text);

    $arrResult = array();

    $pTitle = '#<title>(.*)</title>#isU';
    preg_match_all($pTitle, $content, $arrTitle);
    $strTitle = $arrTitle[1][0];
    $arrTitleSplitter = array('-', '_', '|', ',');
    foreach ($arrTitleSplitter as $strSplitter) {
        if (strpos($strTitle, $strSplitter)) {
            $arrTitle = explode($strSplitter, $strTitle);
            $strTitle = $arrTitle[0];
        }
    }

    $arrResult['title'] = HTMLExtractor::convertToUTF8(trim($strTitle));

    if (!$text) {
        log::w('Get Contents Failed! RequestUrl:' . $url);
        return false;
    }
    $arrResult['body'] = trim(spiderReplaceContent($text));
    $arrResult['body'] = str_replace("\t", ' ', $arrResult['body']);
    $arrResult['body'] = str_replace('  ', ' ', $arrResult['body']);

    $arrResult['body'] = str_replace("　", " ", $arrResult['body']);
    $arrResult['body'] = preg_replace("/&([^;&]*)(;|&)/", "", $arrResult['body']);
    $arrResult['body'] = preg_replace("/[ ]+/s", " ", $arrResult['body']);

    $arrResult['body'] = strip_tags($arrResult['body'], '<br> <hr> <p> <strong> <b> <i> <u>');

    return$arrResult;
}

function getSnoopy($url, $boolhost = true) {
    $ua = array(
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
        '(compatible; MSIE 7.01; MSN 2.5; AOL 4.0; Windows XP)',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11',
        'Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)',
        'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
        'Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)',
    );
    $uriinfo = parse_url($url);
    $snoopy = new Snoopy;
    if ($boolhost) {
        $snoopy->host = $uriinfo['host'];
    }
    $snoopy->agent = $ua[mt_rand(0, count($ua) - 1)];
    $snoopy->referer = $uriinfo['scheme'] . '://' . $uriinfo['host'] . '/';
    $snoopy->rawheaders["Pragma"] = "no-cache";
    if ($boolhost) {
        $snoopy->rawheaders["Host"] = $uriinfo['host'];
    }
    $snoopy->rawheaders["X_FORWARDED_FOR"] = mt_rand(1, 255) . '.' . mt_rand(1, 255) . '.' . mt_rand(1, 255) . '.' . mt_rand(1, 255);
    return $snoopy;
}

function snoopy_parse($snoopy, $url, $method = 'fetch') {
    global $http_status_code;
    $http_status_code = $snoopy->status;
    $logString = __FUNCTION__;
    logformat($logString . '|HttpStatusCode:' . $http_status_code);
    //加入自动重试功能
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        $snoopy->$method($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(5);
        $snoopy->$method($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('Sleep 5 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->$method($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 200) {
        return $snoopy->results;
    }
    return false;
}

function snoopy_fetch($url, $boolhost = true, $ping=false) {
    $intTimeStart = microtime(true);
    $logString = 'NOTICE:Fetching:' . $url;
    global $http_status_code;
    $ua = array(
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
        '(compatible; MSIE 7.01; MSN 2.5; AOL 4.0; Windows XP)',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11',
        'Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)',
        'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
        'Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)',
    );
    $uriinfo = parse_url($url);
    $snoopy = new Snoopy;
    $snoopy->agent = $ua[mt_rand(0, count($ua) - 1)];
    if ($boolhost) {
        $snoopy->host = $uriinfo['host'];
    }
    $snoopy->referer = $uriinfo['scheme'] . '://' . $uriinfo['host'] . '/';
    $snoopy->rawheaders["Pragma"] = "no-cache";
    if ($boolhost) {
        $snoopy->rawheaders["Host"] = $uriinfo['host'];
    }
    $snoopy->rawheaders["X_FORWARDED_FOR"] = mt_rand(1, 255) . '.' . mt_rand(1, 255) . '.' . mt_rand(1, 255) . '.' . mt_rand(1, 255);
    $snoopy->fetch($url);
    $s = $snoopy->results;
    $http_status_code = $snoopy->status;
    $intTimeEnd = microtime(true);
    $logString.='|RequestTime:' . ($intTimeEnd - $intTimeStart) . 's';
    logformat($logString . '|HttpStatusCode:' . $http_status_code);
	if($ping){return $ping;}
    //加入自动重试功能
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        $intTimeEnd = microtime(true);
        $logString.='|RequestTime:' . ($intTimeEnd - $intTimeStart) . 's';
        logformat('ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(5);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        $intTimeEnd = microtime(true);
        $logString.='|RequestTime:' . ($intTimeEnd - $intTimeStart) . 's';
        logformat('Sleep 5 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        $intTimeEnd = microtime(true);
        $logString.='|RequestTime:' . ($intTimeEnd - $intTimeStart) . 's';
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        $intTimeEnd = microtime(true);
        $logString.='|RequestTime:' . ($intTimeEnd - $intTimeStart) . 's';
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        $intTimeEnd = microtime(true);
        $logString.='|RequestTime:' . ($intTimeEnd - $intTimeStart) . 's';
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        $intTimeEnd = microtime(true);
        $logString.='|RequestTime:' . ($intTimeEnd - $intTimeStart) . 's';
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        $intTimeEnd = microtime(true);
        $logString.='|RequestTime:' . ($intTimeEnd - $intTimeStart) . 's';
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }
    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        sleep(10);
        $snoopy->fetch($url);
        $s = $snoopy->results;
        $http_status_code = $snoopy->status;
        $intTimeEnd = microtime(true);
        $logString.='|RequestTime:' . ($intTimeEnd - $intTimeStart) . 's';
        logformat('Sleep 10 Seconds...ReTrying...|' . $logString . '|HttpStatusCode:' . $http_status_code);
    }

    if ($http_status_code == 10060 || '12646132' == $http_status_code || 0 == $http_status_code || substr($http_status_code, 0, 1) == '5') {
        $time = time();
        $lockFile = APP_DATA_PATH . 'ErrorEmailSendLock.txt';
        if (file_exists($lockFile)) {
            $preTime = file_get_contents($lockFile);
        } else {
            $preTime = 0;
        }
		log::n('$lockFile:'.$lockFile);
        if ($time - $preTime > 3600) {
            //$sr=post_email(__FUNCTION__, $url . PHP_EOL . '$http_status_code:' . $http_status_code);
			//log::dw('email send result:'.var_dump($sr));
			log::n('$preTime:'.date('Y-m-d H:i:s',$preTime));
            //log::error('Send Warning Email');
            file_put_contents($lockFile, $time);
            if (defined('SEND_ERROR_SMS') && defined('SEND_ERROR_MOBILE')) {
				log::n('SEND_ERROR_SMS:'.SEND_ERROR_SMS);
				log::n('SEND_ERROR_MOBILE:'.SEND_ERROR_MOBILE);
                send_sms(SEND_ERROR_MOBILE, 'Spider Error,HttpCode:' . $http_status_code . ',URL:' . $url);
            }
        }
        return false;
    }

    global $dhttp_response_header;
    global $dhttp_lastredirectaddr;
    $dhttp_response_header = $http_response_header;
    $dhttp_lastredirectaddr = $snoopy->lastredirectaddr;

    return $s;
}

function dede_fetch($url) {
    $logString = 'Fetching:' . $url;
    $o = new DedeHttpDown;
    $o->OpenUrl($url);
    $s = $o->GetHtml();
    if (!$s) {
        $o->OpenUrl($url);
        $s = $o->GetHtml();
        log::notice('ReTrying...|' . $logString . '|' . $o->m_error);
    }
    if (!$s) {
        sleep(2);
        $o->OpenUrl($url);
        $s = $o->GetHtml();
        log::notice('ReTrying...|Sleep 2:' . $logString . '|' . $o->m_error);
    }
    if (!$s) {
        sleep(5);
        $o->OpenUrl($url);
        $s = $o->GetHtml();
        log::notice('ReTrying...|Sleep 5' . $logString . '|' . $o->m_error);
    }
    if (!$s) {
        sleep(15);
        $o->OpenUrl($url);
        $s = $o->GetHtml();
        log::notice('ReTrying...|Sleep 15' . $logString . '|' . $o->m_error);
    }
    if (!$s) {
        sleep(15);
        $o->OpenUrl($url);
        $s = $o->GetHtml();
        log::notice('ReTrying...|Sleep 15' . $logString . '|' . $o->m_error);
    }
    if (!$s) {
        log::dw('Http Maybe Failed|' . $logString . '|' . $o->m_error);
    }
    if ($s) {
        log::notice($logString . '|Success');
    }
    return $s;
}

function curl_post($post_url, $data, $flag = false) {
    $post = http_build_query($data);
    $login = curl_init($post_url); //创建CURL对象  
    curl_setopt($login, CURLOPT_HEADER, 0); //返回头部  
    curl_setopt($login, CURLOPT_RETURNTRANSFER, 1); //返回信息  
    curl_setopt($login, CURLOPT_POST, 1); //设置POST提交  
    curl_setopt($login, CURLOPT_POSTFIELDS, $post); //提交POST数据  
    curl_setopt($login, CURLOPT_FOLLOWLOCATION, 1);
    curl_setopt($login, CURLOPT_TIMEOUT, 30);

    $data = curl_exec($login); //执行已经定义的设置  
    if ($curl_errno = curl_errno($login)) {
        log::w('curl_errno():' . $curl_errno);
        log::w('curl_error:' . curl_error($login));
        if (!$flag) {
            log::w('sleep 5 and retrying...:');
            sleep(5);
            return curl_post($post_url, $data, true);
        }
    }
    curl_close($login); //关闭 */ 
    return $data;
}

function post($post_url, $data) {
    return curl_post($post_url, $data);
}

/// start ping.inc.php ///

$g_icmp_error = "No Error";

// timeout in ms
function ping($host, $timeout = 30) {
    $port = 0;
    $datasize = 64;
    global $g_icmp_error;
    $g_icmp_error = "No Error";
    $ident = array(ord('J'), ord('C'));
    $seq = array(rand(0, 255), rand(0, 255));

    $packet = '';
    $packet .= chr(8); // type = 8 : request
    $packet .= chr(0); // code = 0

    $packet .= chr(0); // checksum init
    $packet .= chr(0); // checksum init

    $packet .= chr($ident[0]); // identifier
    $packet .= chr($ident[1]); // identifier

    $packet .= chr($seq[0]); // seq
    $packet .= chr($seq[1]); // seq

    for ($i = 0; $i < $datasize; $i++)
        $packet .= chr(0);

    $chk = icmpChecksum($packet);

    $packet[2] = $chk[0]; // checksum init
    $packet[3] = $chk[1]; // checksum init

    $sock = socket_create(AF_INET, SOCK_RAW, getprotobyname('icmp'));
    $time_start = microtime();
    socket_sendto($sock, $packet, strlen($packet), 0, $host, $port);


    $read = array($sock);
    $write = NULL;
    $except = NULL;

    $select = socket_select($read, $write, $except, 0, $timeout * 1000);
    if ($select === NULL) {
        $g_icmp_error = "Select Error";
        socket_close($sock);
        return -1;
    } elseif ($select === 0) {
        $g_icmp_error = "Timeout";
        socket_close($sock);
        return -1;
    }

    $recv = '';
    $time_stop = microtime();
    socket_recvfrom($sock, $recv, 65535, 0, $host, $port);
    $recv = unpack('C*', $recv);

    if ($recv[10] !== 1) { // ICMP proto = 1
        $g_icmp_error = "Not ICMP packet";
        socket_close($sock);
        return -1;
    }

    if ($recv[21] !== 0) { // ICMP response = 0
        $g_icmp_error = "Not ICMP response";
        socket_close($sock);
        return -1;
    }

    if ($ident[0] !== $recv[25] || $ident[1] !== $recv[26]) {
        $g_icmp_error = "Bad identification number";
        socket_close($sock);
        return -1;
    }

    if ($seq[0] !== $recv[27] || $seq[1] !== $recv[28]) {
        $g_icmp_error = "Bad sequence number";
        socket_close($sock);
        return -1;
    }

    $ms = ($time_stop - $time_start) * 1000;

    if ($ms < 0) {
        $g_icmp_error = "Response too long";
        $ms = -1;
    }

    socket_close($sock);

    snoopy_fetch('http://' . $host . '/',false,true);
    global $http_status_code;
    if (!in_array($http_status_code, array(200, 500))) {
        return -1;
    }

    return $ms;
}

function icmpChecksum($data) {
    $bit = unpack('n*', $data);
    $sum = array_sum($bit);

    if (strlen($data) % 2) {
        $temp = unpack('C*', $data[strlen($data) - 1]);
        $sum += $temp[1];
    }

    $sum = ($sum >> 16) + ($sum & 0xffff);
    $sum += ($sum >> 16);

    return pack('n*', ~$sum);
}

function getLastIcmpError() {
    global $g_icmp_error;
    return $g_icmp_error;
}

?>